San Francisco Housing Cost Analysis¶

# imports
import panel as pn
pn.extension('plotly')
import plotly.express as px
import pandas as pd
import hvplot.pandas
import matplotlib.pyplot as plt
import numpy as np
import os
from pathlib import Path
from dotenv import load_dotenv

import warnings
warnings.filterwarnings('ignore')

# Read the census data into a Pandas DataFrame
file_path = Path("Data/sfo_neighborhoods_census_data.csv")
sfo_data = pd.read_csv(file_path, index_col="year")

sfo_data.head()

# Calculate the mean number of housing units per year (hint: use groupby) 

data_group_per_year = sfo_data["housing_units"].groupby(sfo_data.index).mean()
data_group_per_year

# Use the Pandas plot function to plot the average housing units per year.
fig_housing_units = plt.figure()
plot_housing_units = data_group_per_year.plot.bar()
plot_housing_units.set_xlabel('Year', fontsize=12)
plot_housing_units.set_ylabel('Housing Units', fontsize=12)
plot_housing_units.set_title("Housing Units in San Francisco from 2010 to 2016", fontsize=15, fontweight='bold')

housing_units_std =data_group_per_year.std()
housing_units_min =min(data_group_per_year)
housing_units_max =max(data_group_per_year)

plot_housing_units.set_ylim(housing_units_min -housing_units_std, housing_units_max + housing_units_std)
plt.show()
plt.close(fig_housing_units)

# Calculate the average sale price per square foot and average gross rent

Average_of_housing_Cost_per_year = sfo_data[["sale_price_sqr_foot","gross_rent"]].groupby(sfo_data.index).mean()

Average_of_housing_Cost_per_year

# Save the dataframe as a csv file
Average_of_housing_Cost_per_year.reset_index(inplace=True)
Average_of_housing_Cost_per_year.to_csv("Average_of_housing_Cost_per_year.csv",header=True,index=False)
Average_of_housing_Cost_per_year

# Create two line charts, one to plot the average sale price per square foot and another for average montly rent


# Line chart for average sale price per square foot
fig_Average_of_housing_Cost_per_year = plt.figure()
plot_Average_of_housing_Cost_per_year = Average_of_housing_Cost_per_year['sale_price_sqr_foot'].plot(color = 'purple')
plot_Average_of_housing_Cost_per_year.set_xlabel('Year', fontsize=12)
plot_Average_of_housing_Cost_per_year.set_ylabel('Price per SqFt', fontsize=12)
plot_Average_of_housing_Cost_per_year.set_title("Average Sale Price per sqrfoot by Year, San Francisco", fontsize=15, fontweight='bold');


# Line chart for average montly rent
fig_Average_of_housing_Cost_per_year = plt.figure()
plot_Average_of_housing_Cost_per_year = Average_of_housing_Cost_per_year['gross_rent'].plot(color = 'red')
plot_Average_of_housing_Cost_per_year.set_xlabel('Year', fontsize=12)
plot_Average_of_housing_Cost_per_year.set_ylabel('Gross Rent', fontsize=12)
plot_Average_of_housing_Cost_per_year.set_title("Average Gross Rent by Year, San Francisco", fontsize=15, fontweight='bold');

# Group by year and neighborhood and then create a new dataframe of the mean values

df_costs = sfo_data.groupby(["year","neighborhood"]).mean()
df_costs.reset_index(inplace=True)
df_costs.rename(columns={"level_0":"year"}, inplace=True)
df_costs.head(10)

# Use hvplot to create an interactive line chart of the average price per sq ft.
# The plot should have a dropdown selector for the neighborhood
df_costs.hvplot.line(x='year',  y=['sale_price_sqr_foot'],
                          groupby=(['neighborhood']),
                          rot=90).opts(yformatter="%.0f", 
                                       title="Average Price per sq ft.",
                                       ylabel = 'Average sale_price_sqr_foot',
                                       xlabel='Year')

# Use hvplot to create an interactive line chart of the average monthly rent.
# The plot should have a dropdown selector for the neighborhood
df_costs.hvplot.line(x='year',  y=['gross_rent'],
                          groupby=(['neighborhood']),
                          rot=90).opts(yformatter="%.0f", 
                                       title="Average gross_rent.",
                                       ylabel = 'Average gross_rent',
                                       xlabel='Year')

top_ten

# Plotting the data from the top 10 expensive neighborhoods

#top_10_exp_neighborhood = top_ten.nlargest(10,['sale_price_sqr_foot'])
top_ten.hvplot.bar(x='neighborhood',y='sale_price_sqr_foot',ylim=(500.0000 , 950.0000), 
                   rot=90).opts(yformatter="%.0f",
                                title="The top 10 Most Expensive Neighborhoodsin San Francisco from 2010 to 2016")

# Fetch the previously generated DataFrame that was grouped by year and neighborhood
df_costs.head(10)

# Plotting the data from the top 10 expensive neighborhood
expensive_neighborhood_year = (sfo_data.groupby("neighborhood").mean().sort_values("sale_price_sqr_foot", ascending=False))
top_ten_year=expensive_neighborhood_data.head(10)
top_ten_year.reset_index(inplace=True)
top_ten_year                                                           

df_costs.hvplot.bar('year',  y=['gross_rent','sale_price_sqr_foot'],
                          groupby=(['neighborhood']),
                          rot=90).opts(yformatter="%.0f", 
                                       title="Top 10 Expensive Neighborhood in San Francisco from 2010 to 2016",
                                       ylabel = 'Housing Units',
                                       xlabel='Neighborhood')
                                                               

# Load neighborhoods coordinates data
file_path2 = Path("Data/neighborhoods_coordinates.csv")
coord_data = pd.read_csv(file_path2)

coord_data.head()

# Calculate the mean values for each neighborhood

neighborhoods_mean = df_costs.groupby(by=["neighborhood"]).mean().drop(["year"], axis=1)
neighborhoods_mean.reset_index(inplace=True)
neighborhoods_mean.head()

# Join the average values with the neighborhood locations
combined_df = pd.concat([coord_data , neighborhoods_mean], axis="columns", join="inner")
combined_df.head()

# Set the mapbox access token
map_box = px.scatter_mapbox(
    combined_df,
    lat="Lat",
    lon="Lon",
    size="sale_price_sqr_foot",
    color="gross_rent",
    zoom=4,
    title= "Average Values per Neighborhood in San Francisco"
)
map_box.update_layout(
    mapbox_style="carto-positron")
map_box.show() #display map

# Create a scatter mapbox to analyze neighborhood info
# YOUR CODE HERE!

# Fetch the data from all expensive neighborhoods per year.
most_expensive=df_costs[df_costs["neighborhood"].isin(top_ten["neighborhood"])]
most_expensive.head()

# Parallel Categories Plot
px.parallel_categories(
    top_ten,
    dimensions=["neighborhood", "housing_units", "gross_rent"], 
    color="sale_price_sqr_foot",
    color_continuous_scale=px.colors.sequential.Inferno,
    title="Parallel Categories Plot of Most Expensive Neighborhoods in San Francisco per year",
    labels={
        "type": "Neighborhood",
        "region": "Housing Units",
        "prop_size": "Gross Rent",
    },
)

# Parallel Coordinates Plot
px.parallel_coordinates(top_ten, color='sale_price_sqr_foot', 
                        title="Parallel Categories Plot of Most Expensive Neighborhoods in San Francisco per year")

# Sunburst Plot
#df = px.most_expensive.tips() 
  
fig = px.sunburst(most_expensive, path=['year', 'neighborhood'],  
                  values='gross_rent', color='gross_rent',
                  color_continuous_scale=px.colors.sequential.Blues,
                  height=600,
                 title="Sunburst Chart of Most Expensive Neighborhoods in San Francisco per year") 
fig.show()